import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

import static java.lang.System.out;

public class Program {
    static String downloadPage(String link) throws IOException {
        URL url = new URL(link);
        URLConnection urlc = url.openConnection();
        urlc.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16");
        urlc.setRequestProperty("Accept", "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
        urlc.setRequestProperty("Accept-Charset", "utf-8");
        urlc.setRequestProperty("Accept-Language", "en-US,en;q=0.8,ru;q=0.6");
        urlc.setRequestProperty("Cookie", "yandexuid=1830210221301283676; Yandex-CS-Captcha-Evidence=178.130.2.152:1301283713:b9fabdfaa73ff5222034db1e7df41d15");

        InputStreamReader reader = new InputStreamReader(urlc.getInputStream(), "UTF-8");
        StringBuilder builder = new StringBuilder();
        int byteRead;
        while ((byteRead = reader.read()) != -1)
            builder.append((char) byteRead);
        reader.close();
        return builder.toString();
    }


    public static void main(String[] args) throws IOException, UnsupportedEncodingException {

        FileOutputStream fileStream = new FileOutputStream("C:\\rystsov\\Repositories\\Yandex\\Data\\queries.txt", true);
        OutputStreamWriter writer = new OutputStreamWriter(fileStream,"UTF-8");

        int counter = 50;

        String link = "http://stat.yandex.ru/queries/last20.xml";

        while (counter>0 && link!=null) {
            String html = downloadPage(link);
            Document doc = Jsoup.parse(html, link);
            link = null;
            Elements links = doc.select("a[target=_blank]");
            for(Element element : links){
                if (element.hasText()){
                    writer.write(element.text() + "\n");
                }
            }
            Elements next = doc.select("a[style=color:#000000;]");
            for(Element element : next){
                link = "http://stat.yandex.ru" + element.attr("href");
            }
            counter--;
        }

        writer.flush();
        writer.close();
        fileStream.flush();
        fileStream.close();
    }
}
